package com.haiqi.modules.film.schedule.film2018;

import com.haiqi.modules.film.mapper.FilmDao;
import com.haiqi.modules.film.model.FilmEntity;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

/**
 * Created by hythzx on 2016/8/12.
 * 对电影的详细信息进行挖掘
 */
@Component
public class FilmDetailSpider implements PageProcessor {

    Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    @Autowired
    private FilmDao filmDao;

    private Logger logger = Logger.getLogger(this.getClass());

    @Override
    public void process(Page page) {
        Selectable titleXpath = page.getHtml().xpath("//div[@class='title_all']/h1/text()"); //电影标题
        Selectable rankXpath = page.getHtml().xpath("strong[@class='rank']/text()");  //评分
        Selectable updatetimeXpath = page.getHtml().xpath("//span[@class='updatetime']/text()");  // 更新时间
        Selectable urlPath = page.getHtml().xpath("//a/@href").regex("ftp:.*"); //下载地址
        Selectable url = page.getUrl();
        if(StringUtils.isBlank(titleXpath.get())){ //没有找到电影标题，跳过
            return;
        }
        FilmEntity filmEntity1 = filmDao.selectByPrimaryKey(url.get());
        if(filmEntity1 != null){  //已经存在这部电影，跳过
            return;
        }
        logger.info("挖掘到新电影，电影名称是：" + titleXpath.get());
        FilmEntity filmEntity = new FilmEntity();
        filmEntity.setDownloadLink(urlPath.get());
        filmEntity.setFilmName(titleXpath.get());
        filmEntity.setHref(url.get());
        filmEntity.setRank(rankXpath.get());
        filmEntity.setUpdateTime(updatetimeXpath.get());
        filmDao.insertSelective(filmEntity);
    }

    @Override
    public Site getSite() {
        return site;
    }

    public void run(String url){
        logger.info("挖掘地址:" + url);
        Spider.create(this).addUrl(url).thread(5).run();
    }

    public static void main(String[] args) {
        Spider.create(new FilmDetailSpider()).addUrl("http://www.dy2018.com/2/").thread(5).run();
    }
}
